Brain Tumor Classification Pipeline

Imports

# system modules
import os
import itertools
from PIL import Image

# preprocessing modules
import pandas as pd
import numpy as np 
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('whitegrid')

import cv2

from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix

# deep learning modules
import tensorflow as tf
from tensorflow import keras

from tensorflow.keras.models import Sequential
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Activation
from tensorflow.keras.optimizers import Adam, Adamax

#import warnings
#warnings.filterwarnings('ignore')

Preprocessing

Training Data

# path to training data directory
training_data_path = os.path.normpath("brain_tumor_data/Training/")

# initialize lists to store paths of images and their labels
img_paths = []
labels = []

# directories -> lists
training_dir = os.listdir(training_data_path)

# get paths and labels of classes and images in training directory
for i in training_dir:
    class_path = os.path.join(training_data_path, i)
    img_list = os.listdir(class_path)

    for img in img_list:
        img_path = os.path.join(class_path, img)
        img_paths.append(img_path)
        labels.append(i)

# convert lists of img_Paths and their labels into Pandas Series
paths = pd.Series(img_paths, name="path")
labels = pd.Series(labels, name="label")

# concatenate into one Pandas Dataframe
train_data = pd.concat([paths, labels], axis=1)

train_data.shape

(5712, 2)

train_data.head()

	path	label
0	brain_tumor_data\Training\glioma\Tr-glTr_0000.jpg	glioma
1	brain_tumor_data\Training\glioma\Tr-glTr_0001.jpg	glioma
2	brain_tumor_data\Training\glioma\Tr-glTr_0002.jpg	glioma
3	brain_tumor_data\Training\glioma\Tr-glTr_0003.jpg	glioma
4	brain_tumor_data\Training\glioma\Tr-glTr_0004.jpg	glioma

Test Data

# path to test data directory
testing_data_path = os.path.normpath("brain_tumor_data/Testing/")

# initialize lists to store paths of images and their labels
img_paths = []
labels = []

# directories -> lists
testing_dir = os.listdir(testing_data_path)

# get paths and labels of classes and images in testing directory
for i in testing_dir:
    class_path = os.path.join(testing_data_path, i)
    img_list = os.listdir(class_path)

    for img in img_list:
        img_path = os.path.join(class_path, img)
        img_paths.append(img_path)
        labels.append(i)

# convert lists of img_Paths and their labels into Pandas Series
paths = pd.Series(img_paths, name="path")
labels = pd.Series(labels, name="label")

# concatenate into one Pandas Dataframe
test_data = pd.concat([paths, labels], axis=1)

test_data.shape

(1311, 2)

test_data.head()

	path	label
0	brain_tumor_data\Testing\glioma\Te-glTr_0000.jpg	glioma
1	brain_tumor_data\Testing\glioma\Te-glTr_0001.jpg	glioma
2	brain_tumor_data\Testing\glioma\Te-glTr_0002.jpg	glioma
3	brain_tumor_data\Testing\glioma\Te-glTr_0003.jpg	glioma
4	brain_tumor_data\Testing\glioma\Te-glTr_0004.jpg	glioma

Split Test Data into Validation and Test Set

# split test data in half to a validation and test set
# shuffled since originally ordered by tumor classification
valid_df, test_df = train_test_split(test_data, train_size = 0.5, shuffle = True, random_state = 123)
print("Test Set shape:", test_df.shape)
print("Validation Set shape:", valid_df.shape)### Split Test Data into Validation and Test Set

Test Set shape: (656, 2)
Validation Set shape: (655, 2)

Create Image Generators

# hyperparameters
batch_size = 20
img_size = (224,224)
channels = 3
img_shape = (img_size[0], img_size[1], channels)

# initialize generators
training_generator = ImageDataGenerator(fill_mode='nearest')
validation_generator = ImageDataGenerator()
testing_generator = ImageDataGenerator()

Generate New Data for Fitting into Model

train = training_generator.flow_from_dataframe(train_data, x_col = 'path', y_col = 'label', target_size = img_size, class_mode = 'categorical', color_mode = 'rgb', shuffle = True, batch_size = batch_size)

Found 5712 validated image filenames belonging to 4 classes.

valid = validation_generator.flow_from_dataframe(valid_df, x_col = 'path', y_col = 'label', target_size = img_size, class_mode = 'categorical', color_mode = 'rgb', shuffle = True, batch_size = batch_size)

Found 655 validated image filenames belonging to 4 classes.

test = testing_generator.flow_from_dataframe(test_df, x_col = 'path', y_col = 'label', target_size = img_size, class_mode = 'categorical', color_mode = 'rgb', shuffle = False, batch_size = batch_size)

Found 656 validated image filenames belonging to 4 classes.

Sample Batch from Training Data

# define labels and their indices in a dictionary
label_index = train.class_indices
label_index

{'glioma': 0, 'meningioma': 1, 'notumor': 2, 'pituitary': 3}

# label list
keys = list(label_index.keys())
keys

['glioma', 'meningioma', 'notumor', 'pituitary']

# get a sample batch
imgs, labels = next(train)

# visualize the batch
plt.figure(figsize= (20, 20))

for i in range(9):
    plt.subplot(3, 3, i +1)
    im = imgs[i]/255
    plt.imshow(im)
    
    index = np.argmax(labels[i])
    label = keys[index]
    plt.title(label, size=40, color = 'black')
    plt.axis('off')
    
plt.tight_layout()    
plt.show()

Sample Batch from Test Data

# use the same label_index and keys

# get a sample batch
imgs, labels = next(test)

# visualize the batch
plt.figure(figsize= (20, 20))

for i in range(9):
    plt.subplot(3, 3, i +1)
    im = imgs[i]/255
    plt.imshow(im)
    
    index = np.argmax(labels[i])
    label = keys[index]
    plt.title(label, size=40, color = 'black')
    plt.axis('off')
    
plt.tight_layout()    
plt.show()

Sample Batch from Validation Data

# use the same label_index and keys

# get a sample batch
imgs, labels = next(test)

# visualize the batch
plt.figure(figsize= (20, 20))

for i in range(9):
    plt.subplot(3, 3, i +1)
    im = imgs[i]/255
    plt.imshow(im)
    
    index = np.argmax(labels[i])
    label = keys[index]
    plt.title(label, size=40, color = 'black')
    plt.axis('off')
    
plt.tight_layout()    
plt.show()

Model Structure

# hyperparameters for CNN
batch_size = 20
img_size = (224,224)
channels = 3
img_shape = (img_size[0], img_size[1], channels)

# determine the number of unique classes to set the number of neurons in the final layer.
# each neuron represents the probability of a corresponding class
num_classes = len(list(train.class_indices.keys()))
num_classes

4

CNN = Sequential([
    Conv2D(filters = 128, kernel_size = (3,3), padding = 'same', activation = 'elu', input_shape = img_shape),
    Conv2D(filters = 128, kernel_size = (3,3), padding = 'same', activation = 'elu'),
    Conv2D(filters = 128, kernel_size = (3,3), padding = 'same', activation = 'elu'),
    MaxPooling2D((2,2)),
    
    Conv2D(filters = 256, kernel_size = (3,3), padding = 'same', activation = 'elu'),
    Conv2D(filters = 256, kernel_size = (3,3), padding = 'same', activation = 'elu'),
    MaxPooling2D((2,2)),
    
    Conv2D(filters = 256, kernel_size = (3,3), padding = 'same', activation = 'elu'),
    Conv2D(filters = 256, kernel_size = (3,3), padding = 'same', activation = 'elu'),
    MaxPooling2D((2,2)),
    
    Conv2D(filters = 128, kernel_size = (3,3), padding = 'same', activation = 'elu'),
    Conv2D(filters = 128, kernel_size = (3,3), padding = 'same', activation = 'elu'),
    MaxPooling2D((2,2)),
    
    Conv2D(filters = 64, kernel_size = (3,3), padding = 'same', activation = 'elu'),
    Conv2D(filters = 64, kernel_size = (3,3), padding = 'same', activation = 'elu'),
    MaxPooling2D((2,2)),
    
    Flatten(),
    
    Dense(256, activation = 'elu'),
    Dense(128, activation = 'elu'),
    Dense(64, activation = 'elu'),
    Dense(32, activation = 'elu'),
    Dense(num_classes, activation = 'softmax')
])

# model compilation
CNN.compile(Adamax(learning_rate = 0.001), loss = 'categorical_crossentropy', metrics = 'accuracy')

CNN.summary()

Model: "sequential"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d (Conv2D)             (None, 224, 224, 128)     3584      
                                                                 
 conv2d_1 (Conv2D)           (None, 224, 224, 128)     147584    
                                                                 
 conv2d_2 (Conv2D)           (None, 224, 224, 128)     147584    
                                                                 
 max_pooling2d (MaxPooling2  (None, 112, 112, 128)     0         
 D)                                                              
                                                                 
 conv2d_3 (Conv2D)           (None, 112, 112, 256)     295168    
                                                                 
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d (Conv2D)             (None, 224, 224, 128)     3584      
                                                                 
 conv2d_1 (Conv2D)           (None, 224, 224, 128)     147584    
                                                                 
 conv2d_2 (Conv2D)           (None, 224, 224, 128)     147584    
                                                                 
 max_pooling2d (MaxPooling2  (None, 112, 112, 128)     0         
 D)                                                              
                                                                 
 conv2d_3 (Conv2D)           (None, 112, 112, 256)     295168    
                                                                 
 conv2d_4 (Conv2D)           (None, 112, 112, 256)     590080    
                                                                 
 max_pooling2d_1 (MaxPoolin  (None, 56, 56, 256)       0         
 g2D)                                                            
                                                                 
 conv2d_5 (Conv2D)           (None, 56, 56, 256)       590080    
                                                                 
 conv2d_6 (Conv2D)           (None, 56, 56, 256)       590080    
                                                                 
 max_pooling2d_2 (MaxPoolin  (None, 28, 28, 256)       0         
 g2D)                                                            
                                                                 
 conv2d_7 (Conv2D)           (None, 28, 28, 128)       295040    
                                                                 
 conv2d_8 (Conv2D)           (None, 28, 28, 128)       147584    
                                                                 
 max_pooling2d_3 (MaxPoolin  (None, 14, 14, 128)       0         
 g2D)                                                            
                                                                 
 conv2d_9 (Conv2D)           (None, 14, 14, 64)        73792     
                                                                 
 conv2d_10 (Conv2D)          (None, 14, 14, 64)        36928     
                                                                 
 max_pooling2d_4 (MaxPoolin  (None, 7, 7, 64)          0         
 g2D)                                                            
                                                                 
 flatten (Flatten)           (None, 3136)              0         
                                                                 
 dense (Dense)               (None, 256)               803072    
                                                                 
 dense_1 (Dense)             (None, 128)               32896     
                                                                 
 dense_2 (Dense)             (None, 64)                8256      
                                                                 
 dense_3 (Dense)             (None, 32)                2080      
                                                                 
 dense_4 (Dense)             (None, 4)                 132       
                                                                 
=================================================================
Total params: 3763940 (14.36 MB)
Trainable params: 3763940 (14.36 MB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________

Model Training

epochs = 15

history = CNN.fit(x=train, epochs=epochs, verbose=1, validation_data=valid, shuffle=False)
# 19hr 30min to train

Epoch 1/15
286/286 [==============================] - 4692s 16s/step - loss: 0.7265 - accuracy: 0.7365 - val_loss: 0.5123 - val_accuracy: 0.7802
Epoch 2/15
286/286 [==============================] - 4645s 16s/step - loss: 0.3466 - accuracy: 0.8690 - val_loss: 0.3678 - val_accuracy: 0.8550
Epoch 3/15
286/286 [==============================] - 4704s 16s/step - loss: 0.2665 - accuracy: 0.9016 - val_loss: 0.3197 - val_accuracy: 0.8855
Epoch 4/15
286/286 [==============================] - 4703s 16s/step - loss: 0.2099 - accuracy: 0.9219 - val_loss: 0.2782 - val_accuracy: 0.8901
Epoch 5/15
286/286 [==============================] - 4719s 17s/step - loss: 0.1753 - accuracy: 0.9368 - val_loss: 0.2291 - val_accuracy: 0.9023
Epoch 6/15
286/286 [==============================] - 4634s 16s/step - loss: 0.1550 - accuracy: 0.9417 - val_loss: 0.1921 - val_accuracy: 0.9282
Epoch 7/15
286/286 [==============================] - 4627s 16s/step - loss: 0.1263 - accuracy: 0.9554 - val_loss: 0.3103 - val_accuracy: 0.8779
Epoch 8/15
286/286 [==============================] - 4723s 17s/step - loss: 0.1051 - accuracy: 0.9653 - val_loss: 0.1650 - val_accuracy: 0.9359
Epoch 9/15
286/286 [==============================] - 4664s 16s/step - loss: 0.0626 - accuracy: 0.9772 - val_loss: 0.1217 - val_accuracy: 0.9603
Epoch 10/15
286/286 [==============================] - 4651s 16s/step - loss: 0.0644 - accuracy: 0.9757 - val_loss: 0.2170 - val_accuracy: 0.9328
Epoch 11/15
286/286 [==============================] - 4632s 16s/step - loss: 0.0657 - accuracy: 0.9790 - val_loss: 0.1663 - val_accuracy: 0.9557
Epoch 12/15
286/286 [==============================] - 4628s 16s/step - loss: 0.0533 - accuracy: 0.9818 - val_loss: 0.1581 - val_accuracy: 0.9573
Epoch 13/15
286/286 [==============================] - 4621s 16s/step - loss: 0.0397 - accuracy: 0.9860 - val_loss: 0.2222 - val_accuracy: 0.9557
Epoch 14/15
286/286 [==============================] - 4686s 16s/step - loss: 0.0452 - accuracy: 0.9862 - val_loss: 0.1299 - val_accuracy: 0.9649
Epoch 15/15
286/286 [==============================] - 4696s 16s/step - loss: 0.0217 - accuracy: 0.9928 - val_loss: 0.1544 - val_accuracy: 0.9664

Model Evaluation

Metrics

# accuracy and loss on training data
training_acc = history.history['accuracy']
training_loss = history.history['loss']

# accuracy and loss on validation data
validation_acc = history.history['val_accuracy']
validation_loss = history.history['val_loss']

# highest value of validation accuracy, and index of where it happened
index_acc = np.argmax(validation_acc)
high_validation_acc = validation_acc[index_acc]

# lowed value of validation accuracy, and index of where it happened
index_loss = np.argmin(validation_loss)
low_validation_loss = validation_loss[index_loss]

# number of epochs based on length of training accuracy values
epochs =[]
for i in range(len(training_acc)):
    epochs.append (i+1)

# define best epoch
best_acc = f'Best Epoch ={str(index_acc +1)}'
best_loss = f'Best Epoch ={str(index_loss+1)}'

Visualize

plt.figure(figsize = (16, 8))
plt.style.use('bmh')

plt.subplot(1,2,1)
plt.plot(epochs, training_acc, "b", label = "Training Accuarcy")
plt.plot(epochs, validation_acc, "r", label = "Validation Accuarcy")
plt.scatter(index_acc+1, high_validation_acc, s= 150, color = 'green', label = best_acc)

plt.title("Accuracy: Training vs Valid")
plt. xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()

plt.subplot(1,2,2)
plt.plot(epochs, training_loss, "b", label = "Training Loss")
plt.plot(epochs, validation_loss, "r", label = "Validation Loss")
plt.scatter(index_loss+1, low_validation_loss, s= 150, color = 'green', label = best_loss)

plt.title("Loss: Training vs Validation")
plt. xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()

plt.tight_layout()
plt.show()

Get Scores

train_score = CNN.evaluate(train, verbose = 0)
valid_score = CNN.evaluate(valid, verbose = 0)
test_score = CNN.evaluate(test, verbose = 0)

print('____________________________________________________________________')
print(f'Train Scores:        | Validation Scores:    | Test Scores:')
print(f'    Accuracy: {train_score[1]:.4f} |     Accuracy: {valid_score[1]:.4f}  |     Accuracy: {test_score[1]:.4f}')
print(f'    Loss: {train_score[0]:.4f}     |     Loss: {valid_score[0]:.4f}      |     Loss: {test_score[0]:.4f}')
print('_____________________|_______________________|______________________')

____________________________________________________________________
Train Scores:        | Validation Scores:    | Test Scores:
    Accuracy: 0.9953 |     Accuracy: 0.9664  |     Accuracy: 0.9695
    Loss: 0.0141     |     Loss: 0.1544      |     Loss: 0.1005
_____________________|_______________________|______________________

Model Predictions

predictions = CNN.predict(test)
y_pred = np.argmax(predictions, axis = 1)

print("\nPredictions (Probabilities of Each Class):\n", predictions[:5])  # show first 5 predictions
print("\nPredicted Classes (Highest Probable Class):\n", y_pred[:5])  # show first 5 predicted classes

33/33 [==============================] - 160s 5s/step

Predictions (Probabilities of Each Class):
 [[1.9012693e-04 9.9949086e-01 3.1902792e-04 2.1703174e-08]
 [2.2063558e-13 2.5264212e-06 9.9999750e-01 6.4707517e-14]
 [1.6096601e-06 1.5783068e-06 3.9455917e-06 9.9999285e-01]
 [9.9999988e-01 1.1532829e-07 2.4361631e-08 1.0806219e-09]
 [1.2190736e-02 9.8716342e-01 9.6201175e-06 6.3618156e-04]]

Predicted Classes (Highest Probable Class):
 [1 2 3 0 1]

Confusion Matrix

# Use n. of keys of  Class indices to greate confusion matrix
Test_cl_ind = test.class_indices
 
# Get Keys
classes = list(Test_cl_ind.keys())

cm = confusion_matrix(test.classes, y_pred)

plt.figure(figsize =(8, 8))
plt.imshow(cm, interpolation = 'nearest', cmap = plt.cm.Purples)
plt.title("Confusion Matrix")
plt.colorbar()

tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes,rotation = 45)
plt.yticks(tick_marks, classes)

for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
    if cm[i, j] == 0:
        text_color = 'black'  # black for zero values
    elif i == j:
        text_color = 'white'  # white for diagonal (correct classifications)
    else:
        text_color = 'red'    # red for misclassifications

    plt.text(j, i, cm[i, j],
             horizontalalignment='center',
             verticalalignment='center',
             color=text_color)

plt.grid(False)
plt.tight_layout()
plt.ylabel('Actual')
plt.xlabel('Predicted')
plt.show()

Classification Report (Precision, Recall, F1-Score, Accuracy Metrics)

print(classification_report(test.classes, y_pred, target_names = classes))

              precision    recall  f1-score   support

      glioma       0.99      0.91      0.95       164
  meningioma       0.92      0.97      0.94       150
     notumor       0.99      1.00      0.99       193
   pituitary       0.97      0.99      0.98       149

    accuracy                           0.97       656
   macro avg       0.97      0.97      0.97       656
weighted avg       0.97      0.97      0.97       656

Save Model

CNN.save('models/brain_tumor_cnn_classifier.keras')

Load Model (Test Case)

# load model
CNN = tf.keras.models.load_model("models/brain_tumor_cnn_classifier.keras", compile=False)
CNN.compile(optimizer=tf.keras.optimizers.Adamax(learning_rate=0.001), 
            loss='categorical_crossentropy', 
            metrics=['accuracy'])

# imports if we loaded this independently (inside web app)
from PIL import Image # instead of openCV since simpler tasks
import tensorflow as tf
import numpy as np

# paths to images from each class
image_paths = {
    'glioma': 'brain_tumor_data/Testing/glioma/Te-gl_0025.jpg',
    'meningioma': 'brain_tumor_data/Testing/meningioma/Te-me_0010.jpg',
    'no tumor': 'brain_tumor_data/Testing/notumor/\Te-no_0010.jpg',
    'pituitary': 'brain_tumor_data/Testing/pituitary/Te-pi_0040.jpg'
}

# initializations
plt.figure(figsize=(10, 8))
class_labels = ['glioma', 'meningioma', 'no tumor', 'pituitary']

for i, (label, path) in enumerate(image_paths.items()):
    img = Image.open(path)
    img_resized = img.resize((224, 224))
    img_array = tf.keras.preprocessing.image.img_to_array(img_resized)
    img_array = np.expand_dims(img_array, axis=0)  # expand dimensions to fit model input

    # predict and process the results
    predictions = CNN.predict(img_array, verbose=0) # verbose at 0 to prevent status plot
    score = tf.nn.softmax(predictions[0])
    predicted_class = class_labels[np.argmax(score)]

    # plot
    plt.subplot(2, 2, i + 1)
    plt.imshow(img)
    plt.axis('off')

    # title
    actual_text = f'Actual: {label}'
    predicted_text = f'Predicted: {predicted_class}'
    probability_text = f'Probability (vs other 3 classes): {np.max(score):.2f}'
    
    # title formatting
    spacing = 0.02
    plt.text(0.5, 1.15 + 2*spacing, actual_text, ha='center', va='bottom', transform=plt.gca().transAxes, fontsize='medium', color='black')
    plt.text(0.5, 1.10 + spacing, predicted_text, ha='center', va='bottom', transform=plt.gca().transAxes, fontsize='medium', color='green')
    plt.text(0.5, 1.05, probability_text, ha='center', va='bottom', transform=plt.gca().transAxes, fontsize='medium', color='black')

plt.tight_layout()
plt.show()